{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings;warnings.filterwarnings('ignore')\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('train.csv',lineterminator='\\n')\n",
    "df_test = pd.read_csv('test.csv',lineterminator='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>review</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Jo bhi ap se tou behtar hoon</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>ya Allah meri sister Affia ki madad farma</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Yeh khud chahta a is umar main shadi krna.  ha...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Tc ? Apky mun xe exe alfax achy nae lgty 😒💃</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Good</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID                                             review  label\n",
       "0   1                       Jo bhi ap se tou behtar hoon      0\n",
       "1   2          ya Allah meri sister Affia ki madad farma      1\n",
       "2   3  Yeh khud chahta a is umar main shadi krna.  ha...      0\n",
       "3   4        Tc ? Apky mun xe exe alfax achy nae lgty 😒💃      0\n",
       "4   5                                               Good      1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['label'] = df_train['label'].map({'Negative':0,'Positive':1})\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    3361\n",
       "0    2967\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID        0\n",
       "review    0\n",
       "label     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID        0\n",
       "review    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 'Jo bhi ap se tou behtar hoon', 0],\n",
       "       [2, 'ya Allah meri sister Affia ki madad farma', 1],\n",
       "       [3, 'Yeh khud chahta a is umar main shadi krna.  had ogi', 0],\n",
       "       [4, 'Tc ? Apky mun xe exe alfax achy nae lgty 😒💃', 0]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numpy_array = df_train.as_matrix()\n",
    "numpy_array_test = df_test.as_matrix()\n",
    "numpy_array[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([116, 'Kisi Roz Jo Meri Gali Se Ishq Ka Janaza Nikle '],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numpy_array_test[115]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#two commom ways to clean data\n",
    "def cleaner(word):\n",
    "  word = re.sub(r'\\#\\.', '', word)\n",
    "  word = re.sub(r'\\n', '', word)\n",
    "  word = re.sub(r',', '', word)\n",
    "  word = re.sub(r'\\-', ' ', word)\n",
    "  word = re.sub(r'\\.', '', word)\n",
    "  word = re.sub(r'\\\\', ' ', word)\n",
    "  word = re.sub(r'\\\\x\\.+', '', word)\n",
    "  word = re.sub(r'\\d', '', word)\n",
    "  word = re.sub(r'^_.', '', word)\n",
    "  word = re.sub(r'_', ' ', word)\n",
    "  word = re.sub(r'^ ', '', word)\n",
    "  word = re.sub(r' $', '', word)\n",
    "  word = re.sub(r'\\?', '', word)\n",
    "  word = re.sub(r'é', '', word)\n",
    "  word = re.sub(r'§', '', word)\n",
    "  word = re.sub(r'¦', '', word)\n",
    "  word = re.sub(r'æ', '', word)\n",
    "  word = re.sub(r'\\d+', '', word)\n",
    "  word = re.sub('(.*?)\\d+(.*?)', '', word)\n",
    "  return word.lower()\n",
    "def hashing(word):\n",
    "  word = re.sub(r'ain$', r'ein', word)\n",
    "  word = re.sub(r'ai', r'ae', word)\n",
    "  word = re.sub(r'ay$', r'e', word)\n",
    "  word = re.sub(r'ey$', r'e', word)\n",
    "  word = re.sub(r'ie$', r'y', word)\n",
    "  word = re.sub(r'^es', r'is', word)\n",
    "  word = re.sub(r'a+', r'a', word)\n",
    "  word = re.sub(r'j+', r'j', word)\n",
    "  word = re.sub(r'd+', r'd', word)\n",
    "  word = re.sub(r'u', r'o', word)\n",
    "  word = re.sub(r'o+', r'o', word)\n",
    "  word = re.sub(r'ee+', r'i', word)\n",
    "  if not re.match(r'ar', word):\n",
    "    word = re.sub(r'ar', r'r', word)\n",
    "  word = re.sub(r'iy+', r'i', word)\n",
    "  word = re.sub(r'ih+', r'eh', word)\n",
    "  word = re.sub(r's+', r's', word)\n",
    "  if re.search(r'[rst]y', 'word') and word[-1] != 'y':\n",
    "    word = re.sub(r'y', r'i', word)\n",
    "  if re.search(r'[bcdefghijklmnopqrtuvwxyz]i', word):\n",
    "    word = re.sub(r'i$', r'y', word)\n",
    "  if re.search(r'[acefghijlmnoqrstuvwxyz]h', word):\n",
    "    word = re.sub(r'h', '', word)\n",
    "  word = re.sub(r'k', r'q', word)\n",
    "  return word\n",
    "\n",
    "def array_cleaner(array):\n",
    "  # X = array\n",
    "  X = []\n",
    "  for sentence in array:\n",
    "    clean_sentence = ''\n",
    "    words = sentence.split(' ')\n",
    "    for word in words:\n",
    "      clean_sentence = clean_sentence +' '+ cleaner(word)\n",
    "    X.append(clean_sentence)\n",
    "  return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Yaqoob Memon Ki Phansi Zalimana Ghair Insani Hai 20 Saal Qaid Kaat Chukay Thay Amnesty International Ki Bharti Iqdam Ki Muzammat ',\n",
       "       ' Sabit qadam rehna',\n",
       "       'Good decision on ko bhi aam shehryun ki tarah Huqooq hony chahye',\n",
       "       ..., 'Lanat haramdyo', 'Is episode mein koi bhi hot nai hai',\n",
       "       'Mery liye dua kry Allah pak naseeb achy kry ameen'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = numpy_array_test[:,1]\n",
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test if there are nan \n",
    "counter = 1\n",
    "for sentence in X_test:\n",
    "    try:\n",
    "        words = sentence.split(' ')\n",
    "        counter+=1\n",
    "    except:\n",
    "        print(sentence)\n",
    "        print(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' jo bhi ap se tou behtar hoon',\n",
       " ' ya allah meri sister affia ki madad farma',\n",
       " ' yeh khud chahta a is umar main shadi krna  had ogi',\n",
       " ' tc  apky mun xe exe alfax achy nae lgty 😒💃',\n",
       " ' good']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = numpy_array[:, 1]\n",
    "# Clean X here\n",
    "X_train = array_cleaner(X_train)\n",
    "X_test = array_cleaner(X_test)\n",
    "y_train = numpy_array[:, 2]\n",
    "X_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6328\n",
      "2712\n"
     ]
    }
   ],
   "source": [
    "print(len(X_train))\n",
    "print(len(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 0, 1, 0], dtype=int8)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train = np.array(y_train)\n",
    "y_train = y_train.astype('int8')\n",
    "y_train[:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ngram = 2\n",
    "vectorizer = TfidfVectorizer(sublinear_tf=True,ngram_range=(1, ngram), max_df=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_all = X_train + X_test # Combine both to fit the TFIDF vectorization.\n",
    "lentrain = len(X_train)\n",
    "\n",
    "vectorizer.fit(X_all) # This is the slow part!\n",
    "X_all = vectorizer.transform(X_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['賭easar ul', '鄭h', '鄭h isnan', '鄭pwa', '鄭pwa yani']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9040, 113521)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6328, 113521)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_chuli = X_all[:lentrain] # Separate back into training and test sets. \n",
    "X_test_chuli = X_all[lentrain:]\n",
    "X_train_chuli.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.linear_model import SGDClassifier as SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=2019)\n",
    "oof = np.zeros(X_train_chuli.shape[0])\n",
    "predictions = np.zeros(X_test_chuli.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold :1\n",
      "auc score: 0.87129 \n",
      "Fold :2\n",
      "auc score: 0.84598 \n",
      "Fold :3\n",
      "auc score: 0.84839 \n",
      "Fold :4\n",
      "auc score: 0.86017 \n",
      "Fold :5\n",
      "auc score: 0.85166 \n",
      "Fold :6\n",
      "auc score: 0.82702 \n",
      "Fold :7\n",
      "auc score: 0.87713 \n",
      "Fold :8\n",
      "auc score: 0.83738 \n",
      "Fold :9\n",
      "auc score: 0.85027 \n",
      "Fold :10\n",
      "auc score: 0.86954 \n"
     ]
    }
   ],
   "source": [
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(X_train_chuli, y_train)):\n",
    "    print(\"Fold :{}\".format(fold_ + 1))\n",
    "    trn_data = X_train_chuli[trn_idx]\n",
    "    trn_label= y_train[trn_idx]\n",
    "    val_data = X_train_chuli[val_idx]\n",
    "    val_label= y_train[val_idx]\n",
    "    model_SGD = SGD(alpha=0.00001,random_state = 2, shuffle = True, loss = 'log')                      \n",
    "    model_SGD.fit(trn_data, trn_label) # Fit the model.\n",
    "    print(\"auc score: {:<8.5f}\".format(metrics.roc_auc_score(val_label, model_SGD.predict_proba(val_data)[:,1])))\n",
    "    predictions += model_SGD.predict_proba(X_test_chuli)[:,1] / folds.n_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2712\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.20383698, 0.52023924, 0.91316416, 0.74123612])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(predictions))\n",
    "predictions[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "SGD_output = pd.DataFrame({\"ID\":df_test[\"ID\"], \"Pred\":predictions})\n",
    "SGD_output.to_csv('SGD_new.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
